package com.wontube.smr.bo.sites.youtube.spider;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.UUID;

import org.apache.ibatis.session.SqlSession;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.wontube.client.extension.utils.DateUtils;
import com.wontube.client.extension.utils.NumberUtils;
import com.wontube.client.extension.utils.StringUtils;
import com.wontube.client.extension.utils.net.URLUtils;
import com.wontube.core.dao.mybatis.QueryExample;
import com.wontube.smr.bo.constants.SiteConstants;
import com.wontube.smr.bo.sites.youtube.YoutubeMusicGenres;
import com.wontube.smr.client.model.po.MVChart;
import com.wontube.smr.client.model.po.Movie;
import com.wontube.smr.client.model.po.MovieGenre;
import com.wontube.smr.dao.mapper.MVChartMapper;
import com.wontube.smr.dao.mapper.MovieGenreMapper;
import com.wontube.smr.dao.mapper.MovieMapper;
import com.wontube.smr.pub.DefaultDBHelper;

/**
 * @author tanggf(2012-3-30)
 *
 */
public class YoutubeMusicSpider {
	private final static Logger logger = LoggerFactory.getLogger(YoutubeMusicSpider.class);	
	private final static int CONNECT_READ_TIMEOUT = 30 * 1000;
	private final static String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; rv:13.0) Gecko/20100101 Firefox/13.0.1";
	private final static String REFERRER = "http://www.youtube.com/";
	
	private final static String DOMAIN = "http://www.youtube.com";
	private final static String MUSIC_URL = "http://www.youtube.com/music/";
	
	private final static String MUSIC_URL_FORMAT = "http://www.youtube.com/music/{}/videos?sort=p&flow=grid&view=4&page={}";
	
	private final static String[] MUSIC_CATES = YoutubeMusicGenres.getInnerGenres();	
	
	private final static int MAX_MUSIC_PAEG = 50;
	private final static int MUSIC_PAGE_SIZE = 30;
		
	/**
	 * grab all musics from youtube
	 * @return
	 */
	public static int grabAllMusics(){		
		int count = 0;
		//前面10页平行抓取每个genre分类, 后面再分genre独自抓取
		int page = 1;
		for (; page <= 10; page++) {
			for (String cate : MUSIC_CATES) {
				String url = StringUtils.format(MUSIC_URL_FORMAT, cate, page);
				String genre = YoutubeMusicGenres.remendGenre(cate);
				count += parseAndSaveOnePageMusics(genre, url, count);				
			}
		}
		
		//分genre抓取
		for (String cate : MUSIC_CATES) {
			int usePage = page;
			
			while(true){
				String url = StringUtils.format(MUSIC_URL_FORMAT, cate, usePage++);
				String genre = YoutubeMusicGenres.remendGenre(cate);
				int saved = parseAndSaveOnePageMusics(genre, url, count);	
				count += saved;
				if(saved == 0 || saved < MUSIC_PAGE_SIZE  || usePage >= MAX_MUSIC_PAEG){
					break;
				}				
			}
		}
		
		return count;
	}	

	/**
	 * 抓取youtube daily top music chart
	 */
	public static int grabDailyMusicCharts(){
		int count = 0;
		Document doc = getDocument(MUSIC_URL);		
		if(doc != null){
			String selector = "#daily-hits";			
			List<MVChart> charts = getMusicChartsFromDiv(doc, selector);				
			saveMVChart2db(charts, MVChart.CATEGORY_YOUTUBE_TOP, MVChart.UPDATE_PERIOD_DAILY);
			count = charts.size();
		}
		
		return count;
	}
	
	/**
	 * 抓取youtube_weekly chart and billboard_weekly chart
	 */
	public static int grabWeeklyMusicCharts(){
		int count = 0;
		Document doc = getDocument(MUSIC_URL);		
		if(doc != null){
			//youtube_weekly chart
			String selector = "#weekly-hits";			
			List<MVChart> charts = getMusicChartsFromDiv(doc, selector);			
			saveMVChart2db(charts, MVChart.CATEGORY_YOUTUBE_TOP, MVChart.UPDATE_PERIOD_WEEKLY);
			count += charts.size();
			
			//billboard weekly chart
			selector = "#billboard-hits";			
			charts = getMusicChartsFromDiv(doc, selector);			
			saveMVChart2db(charts, MVChart.CATEGORY_BILLBOARD_TOP, MVChart.UPDATE_PERIOD_WEEKLY);
			count += charts.size();
		}		
		
		return count;
	}

	/**
	 * grab and save one page musics
	 * @param genre
	 * @param lis
	 * @param index
	 * @return
	 */
	private static int parseAndSaveOnePageMusics(String genre, String url, int index) {
		Document doc = getDocument(url);
		if(doc == null){
			return 0;
		}
		
		Elements lis = doc.select("#branded-page-body > div.channel-tab-content > div.tab-content-body > div.primary-pane > div.channel-browse > ul.channels-browse-content-grid > li.channels-content-item");
		if(lis.isEmpty()){
			return 0;
		}
		
		List<Movie> musics = new ArrayList<Movie>();
		for (Element ele : lis) {
			String img = ele.select("a.ux-thumb-wrap > span.video-thumb > span.yt-thumb-clip > span.yt-thumb-clip-inner > img").attr("src");
			if(img.isEmpty()){
				continue;
			}
			
			Movie movie = new Movie();
			movie.setThumbnail(img);
			String time = ele.select("a.ux-thumb-wrap > span.video-time").text().trim();
			movie.setDuration(StringUtils.toSeconds(time));
			String title = ele.select("a.content-item-title").text().trim();
			movie.setTitle(title);
			movie.setTitleInitial(StringUtils.getInitial(title));			
			
			Element ve = ele.select("span.content-item-detail").get(0);		
			if(ve != null){
				String view = ve.ownText().trim();
				view = view.replace("views", "").replace(",", "");				
				movie.setViewCount(NumberUtils.parseInt(view.trim()));
			}
			
			String uploader = ele.select("span.content-item-detail > a.yt-user-name").text().trim();
			movie.setUploader(uploader);
			
			//videoid
			String videoId = URLUtils.parseQueryString(ele.select("a.ux-thumb-wrap").attr("href")).get("v");
			movie.setId(SiteConstants.ID_PREFIX_YOUTUBE + videoId);
			movie.setWebsite(SiteConstants.WEBSITE_YOUTUBE);
			String itemUrl = DOMAIN + "/watch?v=" + videoId;
			movie.setItemUrl(itemUrl);
			movie.setMovieType(Movie.TYPE_MUSIC);
			movie.setVideoId(videoId);
			movie.setGenre(genre);
			
			//singer			
			Document itemDoc = getDocument(itemUrl);
			if(itemDoc != null){
				String singer = itemDoc.select("#watch-description-extra-info > li.full-link span.link-like").text().trim();
				movie.setActors(singer);
			}
			
			musics.add(movie);			
		}		
		
		//save to db
		Date version = DateUtils.getNow();
		SqlSession session = DefaultDBHelper.getBatchSqlSession();
		try {
			MovieMapper mapper = session.getMapper(MovieMapper.class);
			MovieGenreMapper genreMapper = session.getMapper(MovieGenreMapper.class);
			
			for (Movie music : musics) {
				music.setVersion(version);
				music.setSortIndex(++index);
				String movieId = music.getId();
				Movie temp = mapper.selectByPrimaryKey(movieId);
				boolean isGenreRef = false;
				if(temp != null){
					QueryExample ex = new QueryExample();
					ex.or().addCriterion("movie_genre.movie_id=", movieId)
						.addCriterion("movie_genre.genre_name=", genre);
					isGenreRef = genreMapper.countByExample(ex) > 0;
					if( ! isGenreRef){
						String tg = temp.getGenre();
						if( ! StringUtils.isNullOrEmpty(tg)){
							music.setGenre(tg + "," + genre);
						}						
					}
					//update
					mapper.updateByPrimaryKey(music);
				}else{
					//insert					
					mapper.insert(music);
				}
				
				//genre
				if( ! isGenreRef){
					MovieGenre mg = new MovieGenre();
					mg.setId(UUID.randomUUID().toString());
					mg.setGenreName(genre);
					mg.setMovieId(movieId);
					genreMapper.insert(mg);
				}
			}
			
			session.commit();
		} finally {
			session.close();
		}
		
		return musics.size();
	}
	
	/**
	 * 解析音乐排行榜层信息
	 * @param doc
	 * @param selector
	 * @return
	 */
	private static List<MVChart> getMusicChartsFromDiv(Document doc, String selector) {
		List<MVChart> musics = new ArrayList<MVChart>();
		//select到包含音乐信息的<li>标签元素
		Elements liElements = doc.select(selector + " > div.yt-uix-slider > div.yt-uix-slider-body > div.yt-uix-slider-slides > ul.yt-uix-slider-slide > li.yt-uix-slider-slide-item");
		if(liElements.isEmpty()){
			return musics;
		}				
		
		for (Element liEle : liElements) {				
			//循环处理<li>,获取信息 
			MVChart chart = parseMusicInfoInLi(liEle);
			if(chart != null){
				chart.setWebsite(SiteConstants.WEBSITE_YOUTUBE);				
				musics.add(chart);
			}
		}	
	
		return musics;
	}	
	
	/**
	 * 解析取出<li>元素关于视频的信息
	 * @param liEle
	 */
	private static MVChart parseMusicInfoInLi(Element liEle) {
		Elements imgNode = liEle.select("div.browse-item > a.ux-thumb-wrap > span.video-thumb > span.yt-thumb-clip > span.yt-thumb-clip-inner > img");
		if(imgNode.isEmpty()){
			return null;
		}
		
		MVChart chart = new MVChart();
		String imgUrl = imgNode.attr("data-thumb");
		if(StringUtils.isNullOrEmpty(imgUrl)){
			imgUrl = imgNode.attr("src");
		}	
		chart.setThumbnail(imgUrl);
		
		//singer, title, videoId
		Elements aele = liEle.select("div.browse-item > div.browse-item-content > h3 > a");
		String href = aele.attr("href");		
		String videoId = URLUtils.parseQueryString(href).get("v");
		chart.setVideoId(videoId);		
		chart.setItemUrl(DOMAIN + "/watch?v=" + videoId);
		
		chart.setTitle(aele.text().trim());		
		String singer = liEle.select("div.browse-item > div.browse-item-content > span.browse-item-info > a").text().trim();
		chart.setSinger(singer);
		
		return chart;
	}
	
	/**
	 * 批量插入数据
	 * @param charts
	 * @param category
	 * @param period
	 */
	private static void saveMVChart2db(List<MVChart> charts, String category, String period) {		
		int count = 0;
		// 时间戳
		Date now = Calendar.getInstance().getTime();
		// 批量session
		SqlSession session = DefaultDBHelper.getBatchSqlSession();		
		try {		
			MVChartMapper mapper = session.getMapper(MVChartMapper.class);
			for (MVChart chart : charts) {	
				String videoId = chart.getVideoId();
				
				chart.setVersion(now);
				chart.setSortIndex(++count);
				chart.setMovieId(SiteConstants.ID_PREFIX_YOUTUBE  + videoId);
				chart.setCategory(category);
				chart.setUpdatePeriod(period);		
				
				// category, update_period, video_id决定唯一性				
				QueryExample ex = new QueryExample();
				ex.or().addCriterion("mv_chart.video_id=", videoId)
						.addCriterion("mv_chart.category=", category)
						.addCriterion("mv_chart.update_period=", period);
				List<MVChart> musics = mapper.selectByExample(ex);
				
				if (musics.size() > 0) {
					// update					
					chart.setId(musics.get(0).getId());
					mapper.updateByPrimaryKey(chart);
				} else {
					// insert
					chart.setId(UUID.randomUUID().toString());					
					mapper.insert(chart);
				}
			}
			
			session.commit();
		} finally {
			session.close();
		}				
	}
	
	private static Document getDocument(String url) {
		try {
			return Jsoup.connect(url).timeout(CONNECT_READ_TIMEOUT).userAgent(USER_AGENT).referrer(REFERRER).get();
		} catch (IOException e) {
			logger.error("error happened when fetch " + url + " webpage: " + e);
		}
		
		return null;
	}
	
	public static void main(String[] args) {
		grabDailyMusicCharts();
	}
}
